package com.study.spark.scala.rdd

import org.apache.spark.{SparkConf, SparkContext}

/**
  * Checkpoint使用示例
  * @author stephen
  * @create 2019-03-15 09:58
  * @since 1.0.0
  */
object CheckpointDemo {

  def main(args: Array[String]): Unit = {
    val conf = new SparkConf()
      .setMaster("local[*]")
      .setAppName("Checkpoint Demo")
    val sc = new SparkContext(conf)
    // 1.设置checkpoint存放目录
    sc.setCheckpointDir("hdfs://localhost:9000/bigdata-study/ckp")

    val rdd = sc.textFile("hdfs://localhost:9000/bigdata-study/spark")
    // 2.经过复杂计算得到中间结果
    val wordAndOne = rdd.flatMap(_.split(" ")).map((_, 1))
    // 3.中间结果进行保存
    wordAndOne.checkpoint()
    // 4.使用checkpoint数据
    val resultRdd = wordAndOne.reduceByKey(_+_)

    resultRdd.collect()

    sc.stop()
  }
}
